{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/root/toutiao_project/reco_sys\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "# 如果当前代码文件运行测试需要加入修改路径，避免出现后导包问题\n",
    "BASE_DIR = os.path.dirname(os.path.dirname(os.getcwd()))\n",
    "sys.path.insert(0, os.path.join(BASE_DIR))\n",
    "print(BASE_DIR)\n",
    "PYSPARK_PYTHON = \"/miniconda2/envs/reco_sys/bin/python\"\n",
    "# 当存在多个版本时，不指定很可能会导致出错\n",
    "os.environ[\"PYSPARK_PYTHON\"] = PYSPARK_PYTHON\n",
    "os.environ[\"PYSPARK_DRIVER_PYTHON\"] = PYSPARK_PYTHON\n",
    "from offline import SparkSessionBase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OriginArticleData(SparkSessionBase):\n",
    "    \n",
    "    SPARK_APP_NAME = \"mergeArticle\"\n",
    "    SPARK_URL = \"yarn\"\n",
    "\n",
    "    ENABLE_HIVE_SUPPORT = True\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.spark = self._create_spark_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "oa = OriginArticleData()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 进行文章 前两个表 的合并\n",
    "oa.spark.sql(\"use toutiao\")\n",
    "# news_article_basic 与news_article_content, article_id\n",
    "titlce_content = oa.spark.sql(\"select a.article_id, a.channel_id, a.title, b.content from news_article_basic a inner join news_article_content b on a.article_id=b.article_id where a.article_id=116636\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+----------+---------------+--------------------+\n",
      "|article_id|channel_id|          title|             content|\n",
      "+----------+----------+---------------+--------------------+\n",
      "|    116636|        18|动态再平衡投资策略历史数据回测|<p>赚钱是个俗气的话题，但又是人...|\n",
      "+----------+----------+---------------+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "titlce_content.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 进行title_content 与 文章频道名称合并\n",
    "titlce_content.registerTempTable('temptable')\n",
    "\n",
    "channel_title_content = oa.spark.sql(\"select t.*, n.channel_name from temptable t left join news_channel n on t.channel_id=n.channel_id\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+----------+---------------+--------------------+------------+\n",
      "|article_id|channel_id|          title|             content|channel_name|\n",
      "+----------+----------+---------------+--------------------+------------+\n",
      "|    116636|        18|动态再平衡投资策略历史数据回测|<p>赚钱是个俗气的话题，但又是人...|      python|\n",
      "+----------+----------+---------------+--------------------+------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "channel_title_content.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 合并三个内容到一个字符串\n",
    "import pyspark.sql.functions as F\n",
    "\n",
    "sentence_df = channel_title_content.select(\"article_id\", \"channel_id\", \"channel_name\", \"title\", \"content\", \n",
    "                            F.concat_ws(',', \n",
    "                                       channel_title_content.channel_name,\n",
    "                                       channel_title_content.title,\n",
    "                                       channel_title_content.content).alias('sentence'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+----------+------------+---------------+--------------------+--------------------+\n",
      "|article_id|channel_id|channel_name|          title|             content|            sentence|\n",
      "+----------+----------+------------+---------------+--------------------+--------------------+\n",
      "|    116636|        18|      python|动态再平衡投资策略历史数据回测|<p>赚钱是个俗气的话题，但又是人...|python,动态再平衡投资策略历...|\n",
      "+----------+----------+------------+---------------+--------------------+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "sentence_df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+----------+------------+--------------------+--------------------+--------------------+\n",
      "|article_id|channel_id|channel_name|               title|             content|            sentence|\n",
      "+----------+----------+------------+--------------------+--------------------+--------------------+\n",
      "|         1|        17|          前端|     Vue props用法小结原荐|<p><strong>Vue pr...|前端,Vue props用法小结原...|\n",
      "|         2|        17|          前端|vue.js响应式原理解析与实现—...|<p>上次我们已经分析了vue.j...|前端,vue.js响应式原理解析与...|\n",
      "|         3|        17|          前端|JavaScript中浅拷贝和深拷...|<p>要理解 JavaScript...|前端,JavaScript中浅拷贝...|\n",
      "|         4|        17|          前端|基于vue2.0 +vuex+ e...|<p>效果演示地址,</p><p>...|前端,基于vue2.0 +vuex...|\n",
      "|         5|        17|          前端|immutability因Reac...|<p><img src=\"http...|前端,immutability因R...|\n",
      "|         6|        17|          前端|简单了解 node npm cnp...|<span id=\"OSC_h1_...|前端,简单了解 node npm ...|\n",
      "|         7|        17|          前端|       Web工程师以太坊入门原荐|<p>我经常构建使用以太坊的Web...|前端,Web工程师以太坊入门原荐,...|\n",
      "|         8|        17|          前端|Webpack 4 api 了解与...|<h2>webpack 最近升级到...|前端,Webpack 4 api ...|\n",
      "|         9|        17|          前端|vue中用数据驱动视图的思想去写代...|<h4>前言</h4><p>如何在...|前端,vue中用数据驱动视图的思想...|\n",
      "|        10|        17|          前端|微信小程序中如何使用WebSock...|<p>本文由腾讯云技术团队原创，感...|前端,微信小程序中如何使用WebS...|\n",
      "+----------+----------+------------+--------------------+--------------------+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 读取文章，进行每篇张分词\n",
    "oa.spark.sql(\"use article\")\n",
    "article_data = oa.spark.sql(\"select * from article_data limit 10\")\n",
    "article_data.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 文章数据进行分词处理,得到分词结果\n",
    "# 分词\n",
    "def segmentation(partition):\n",
    "    import os\n",
    "    import re\n",
    "\n",
    "    import jieba\n",
    "    import jieba.analyse\n",
    "    import jieba.posseg as pseg\n",
    "    import codecs\n",
    "\n",
    "    abspath = \"/root/words\"\n",
    "\n",
    "    # 结巴加载用户词典\n",
    "    userDict_path = os.path.join(abspath, \"ITKeywords.txt\")\n",
    "    jieba.load_userdict(userDict_path)\n",
    "\n",
    "    # 停用词文本\n",
    "    stopwords_path = os.path.join(abspath, \"stopwords.txt\")\n",
    "\n",
    "    def get_stopwords_list():\n",
    "        \"\"\"返回stopwords列表\"\"\"\n",
    "        stopwords_list = [i.strip()\n",
    "                          for i in codecs.open(stopwords_path).readlines()]\n",
    "        return stopwords_list\n",
    "\n",
    "    # 所有的停用词列表\n",
    "    stopwords_list = get_stopwords_list()\n",
    "\n",
    "    # 分词\n",
    "    def cut_sentence(sentence):\n",
    "        \"\"\"对切割之后的词语进行过滤，去除停用词，保留名词，英文和自定义词库中的词，长度大于2的词\"\"\"\n",
    "        # print(sentence,\"*\"*100)\n",
    "        # eg:[pair('今天', 't'), pair('有', 'd'), pair('雾', 'n'), pair('霾', 'g')]\n",
    "        seg_list = pseg.lcut(sentence)\n",
    "        seg_list = [i for i in seg_list if i.flag not in stopwords_list]\n",
    "        filtered_words_list = []\n",
    "        for seg in seg_list:\n",
    "            # print(seg)\n",
    "            if len(seg.word) <= 1:\n",
    "                continue\n",
    "            elif seg.flag == \"eng\":\n",
    "                if len(seg.word) <= 2:\n",
    "                    continue\n",
    "                else:\n",
    "                    filtered_words_list.append(seg.word)\n",
    "            elif seg.flag.startswith(\"n\"):\n",
    "                filtered_words_list.append(seg.word)\n",
    "            elif seg.flag in [\"x\", \"eng\"]:  # 是自定一个词语或者是英文单词\n",
    "                filtered_words_list.append(seg.word)\n",
    "        return filtered_words_list\n",
    "\n",
    "    for row in partition:\n",
    "        sentence = re.sub(\"<.*?>\", \"\", row.sentence)    # 替换掉标签数据\n",
    "        words = cut_sentence(sentence)\n",
    "        yield row.article_id, row.channel_id, words\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "words_df = article_data.rdd.mapPartitions(segmentation).toDF(['article_id', 'channel_id', 'words'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+----------+--------------------+\n",
      "|article_id|channel_id|               words|\n",
      "+----------+----------+--------------------+\n",
      "|      4273|        15|[javascript, reac...|\n",
      "|      4274|        19|[java, java, 笔记, ...|\n",
      "|      4275|        19|[java, 传统, 方式, 类继...|\n",
      "|      4276|        15|[javascript, Vue,...|\n",
      "|      4278|        15|[javascript, 作用域链...|\n",
      "|      4279|        19|[java, springboot...|\n",
      "|      4280|        19|[java, Jedis, 工具类...|\n",
      "|      4281|        19|[java, java, 记录, ...|\n",
      "|      4282|        15|[javascript, VueS...|\n",
      "|      4283|        15|[javascript, 体积, ...|\n",
      "+----------+----------+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "words_df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 先计算分词之后的每篇文章的词频，得到CV模型\n",
    "# 统计所有文章不同的词，组成一个词列表 words_list = [1,2,3,,34,4,45,56,67,78,8.......,,,,.]\n",
    "from pyspark.ml.feature import CountVectorizer\n",
    "cv = CountVectorizer(inputCol='words', outputCol='countFeatures', vocabSize=2000, minDF=1.0)\n",
    "cv_model = cv.fit(words_df)\n",
    "\n",
    "# 然后根据词频计算IDF以及词，得到IDF模型\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_model.write().overwrite().save(\"hdfs://hadoop-master:9000/headlines/models/test.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import CountVectorizerModel\n",
    "cv_m = CountVectorizerModel.load(\"hdfs://hadoop-master:9000/headlines/models/test.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_result = cv_m.transform(words_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+----------+--------------------+--------------------+\n",
      "|article_id|channel_id|               words|       countFeatures|\n",
      "+----------+----------+--------------------+--------------------+\n",
      "|      4273|        15|[javascript, reac...|(1265,[0,1,3,8,10...|\n",
      "|      4274|        19|[java, java, 笔记, ...|(1265,[24,29,32,3...|\n",
      "|      4275|        19|[java, 传统, 方式, 类继...|(1265,[4,29,30,34...|\n",
      "|      4276|        15|[javascript, Vue,...|(1265,[0,1,8,12,1...|\n",
      "|      4278|        15|[javascript, 作用域链...|(1265,[5,11,12,15...|\n",
      "|      4279|        19|[java, springboot...|(1265,[3,4,21,31,...|\n",
      "|      4280|        19|[java, Jedis, 工具类...|(1265,[1,14,16,18...|\n",
      "|      4281|        19|[java, java, 记录, ...|(1265,[29,30,57,9...|\n",
      "|      4282|        15|[javascript, VueS...|(1265,[1,2,4,9,19...|\n",
      "|      4283|        15|[javascript, 体积, ...|(1265,[0,1,11,17,...|\n",
      "+----------+----------+--------------------+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cv_result.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# IDF 模型\n",
    "from pyspark.ml.feature import IDF\n",
    "idf = IDF(inputCol=\"countFeatures\", outputCol=\"idfFeatures\")\n",
    "idfModel = idf.fit(cv_result)\n",
    "idfModel.write().overwrite().save(\"hdfs://hadoop-master:9000/headlines/models/testIDF.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['this',\n",
       " 'pa',\n",
       " 'node',\n",
       " 'data',\n",
       " '数据',\n",
       " 'let',\n",
       " 'keys',\n",
       " 'obj',\n",
       " '组件',\n",
       " 'npm',\n",
       " 'child',\n",
       " '节点',\n",
       " 'log',\n",
       " '属性',\n",
       " 'key',\n",
       " 'console',\n",
       " 'value',\n",
       " 'var',\n",
       " 'return',\n",
       " 'div',\n",
       " 'name',\n",
       " '文本',\n",
       " 'const',\n",
       " 'fragment',\n",
       " '&#',\n",
       " 'WebSocket',\n",
       " 'props',\n",
       " 'msg',\n",
       " '合约',\n",
       " '对象',\n",
       " '代码',\n",
       " 'ul',\n",
       " '.a',\n",
       " 'for',\n",
       " '程序',\n",
       " 'model',\n",
       " 'web3',\n",
       " '时候',\n",
       " '以太坊',\n",
       " 'forChildMsg',\n",
       " 'obj2',\n",
       " '交易',\n",
       " 'class',\n",
       " 'update',\n",
       " '函数',\n",
       " 'mongodb',\n",
       " 'config',\n",
       " '数组',\n",
       " '版本',\n",
       " '内容',\n",
       " 'amp',\n",
       " 'web',\n",
       " '语法',\n",
       " 'test',\n",
       " 'vue',\n",
       " 'ownChildMsg',\n",
       " '项目',\n",
       " '方法',\n",
       " 'childNode',\n",
       " 'val',\n",
       " 'immutable',\n",
       " 'new',\n",
       " 'function',\n",
       " 'matchs',\n",
       " 'match',\n",
       " 'array',\n",
       " 'template',\n",
       " 'children',\n",
       " '文章',\n",
       " 'DOM',\n",
       " 'Watcher',\n",
       " '域名',\n",
       " 'set',\n",
       " '信息',\n",
       " 'Object',\n",
       " '区块链',\n",
       " '元素节点',\n",
       " 'input',\n",
       " 'attr',\n",
       " '官方',\n",
       " '文件',\n",
       " 'Array',\n",
       " '方式',\n",
       " '命令',\n",
       " '功能',\n",
       " 'install',\n",
       " 'ckage',\n",
       " 'truffle',\n",
       " '服务器',\n",
       " 'newVal',\n",
       " 'textReg',\n",
       " '定义',\n",
       " 'rawTextContent',\n",
       " 'Web',\n",
       " 'item',\n",
       " 'registry',\n",
       " 'JSON',\n",
       " 'forEach',\n",
       " 'textContent',\n",
       " '智能',\n",
       " 'arr',\n",
       " 'gas',\n",
       " '部署',\n",
       " 'r2',\n",
       " '类型',\n",
       " '源码',\n",
       " 'the',\n",
       " 'get',\n",
       " 'init',\n",
       " 'rse',\n",
       " '镜像',\n",
       " 'compile',\n",
       " '用户',\n",
       " 'col',\n",
       " 'mongod',\n",
       " '地址',\n",
       " 'call',\n",
       " 'MVVM',\n",
       " '字符串',\n",
       " 'http',\n",
       " 'eth',\n",
       " 'initDom',\n",
       " 'style',\n",
       " 'xxx',\n",
       " '数据库',\n",
       " '用法',\n",
       " 'json',\n",
       " 'data2',\n",
       " 'from',\n",
       " 'target',\n",
       " 'CompileUtils',\n",
       " 'cli',\n",
       " 'constructor',\n",
       " 'add',\n",
       " 'balance',\n",
       " 'cnpm',\n",
       " '首席',\n",
       " '原始数据',\n",
       " 'rentNode',\n",
       " 'Vue',\n",
       " 'https',\n",
       " 'childNodes',\n",
       " '苏南',\n",
       " 'server',\n",
       " '技术',\n",
       " 'object',\n",
       " '.h',\n",
       " 'firstChild',\n",
       " '游戏',\n",
       " 'nodejs',\n",
       " 'prop',\n",
       " '建议',\n",
       " 'Dep',\n",
       " '参数',\n",
       " '问题',\n",
       " '过程',\n",
       " 'SocketIO',\n",
       " '安装包',\n",
       " '全局',\n",
       " '以太',\n",
       " '文件夹',\n",
       " '情况',\n",
       " 'remove',\n",
       " 'length',\n",
       " '大家',\n",
       " 'newTextContent',\n",
       " 'org',\n",
       " '协议',\n",
       " '菜单',\n",
       " '房间',\n",
       " '拷贝',\n",
       " 'rent',\n",
       " '状态',\n",
       " '总结',\n",
       " 'document',\n",
       " 'String',\n",
       " '指令',\n",
       " '视图',\n",
       " 'isElementNode',\n",
       " 'require',\n",
       " 'icon',\n",
       " '客户端',\n",
       " '时间',\n",
       " '模块',\n",
       " '腾讯',\n",
       " 'backgroundColor',\n",
       " 'index',\n",
       " 'socket',\n",
       " '剪刀',\n",
       " '系统',\n",
       " 'HTTPS',\n",
       " '石头',\n",
       " 'prototype',\n",
       " 'stringify',\n",
       " 'element',\n",
       " 'API',\n",
       " 'nodeType',\n",
       " '例子',\n",
       " 'toString',\n",
       " '开发者',\n",
       " 'accounts',\n",
       " 'apply',\n",
       " '插件',\n",
       " 'rawObj',\n",
       " '子节点',\n",
       " 'save',\n",
       " 'components',\n",
       " 'nodeName',\n",
       " '原因',\n",
       " 'list',\n",
       " 'Number',\n",
       " '即时通讯',\n",
       " 'getModelValue',\n",
       " 'Socket',\n",
       " 'start',\n",
       " 'bin',\n",
       " 'defineProperty',\n",
       " '逻辑',\n",
       " 'node2Fragment',\n",
       " '私钥',\n",
       " 'ppendChild',\n",
       " 'address',\n",
       " 'tougu',\n",
       " 'updateTextNode',\n",
       " 'defineReactive',\n",
       " 'Observer',\n",
       " 'ddr',\n",
       " 'vue2',\n",
       " 'optimization',\n",
       " 'splice',\n",
       " '交流',\n",
       " 'reg',\n",
       " '事件',\n",
       " 'express',\n",
       " '效果',\n",
       " 'process',\n",
       " '帐户',\n",
       " 'watcher',\n",
       " 'click',\n",
       " '模板',\n",
       " '附件',\n",
       " 'windows',\n",
       " '无法',\n",
       " 'css',\n",
       " '模式',\n",
       " 'oldValue',\n",
       " 'taobao',\n",
       " '兴趣',\n",
       " 'subs',\n",
       " 'deepclone',\n",
       " 'mode',\n",
       " '证书',\n",
       " 'sub',\n",
       " 'compileModelAttr',\n",
       " 'sender',\n",
       " 'uint',\n",
       " 'newValue',\n",
       " '入门',\n",
       " '属性值',\n",
       " '平台',\n",
       " 'Node',\n",
       " '服务端',\n",
       " 'original',\n",
       " '特性',\n",
       " '重绘',\n",
       " '记录',\n",
       " 'JavaScript',\n",
       " '警告',\n",
       " '规范',\n",
       " '人员',\n",
       " 'push',\n",
       " 'updateCb',\n",
       " 'nodeJs',\n",
       " 'unshift',\n",
       " '声明',\n",
       " 'ssign',\n",
       " 'nginx',\n",
       " 'merge',\n",
       " 'exports',\n",
       " '深拷贝',\n",
       " 'unset',\n",
       " '余额',\n",
       " 'row',\n",
       " 'ShenZhen',\n",
       " '程序员',\n",
       " 'isDirective',\n",
       " '工具',\n",
       " 'connection',\n",
       " '矿工',\n",
       " 'compileTextNode',\n",
       " 'net',\n",
       " 'hobby',\n",
       " 'uint256',\n",
       " '数据驱动',\n",
       " '全栈',\n",
       " 'type',\n",
       " 'Map',\n",
       " '配置文件',\n",
       " '页面',\n",
       " '常规',\n",
       " 'Mod',\n",
       " 'body',\n",
       " 'validator',\n",
       " 'gasPrice',\n",
       " 'goLink',\n",
       " 'Set',\n",
       " '平头',\n",
       " '栈内存',\n",
       " '形式',\n",
       " 'isTextNode',\n",
       " 'touzi',\n",
       " '瓶颈',\n",
       " 'testrpc',\n",
       " '基础',\n",
       " 'Immutability',\n",
       " '哈希',\n",
       " '动态',\n",
       " 'and',\n",
       " '体验',\n",
       " '思维能力',\n",
       " 'loadStyle',\n",
       " 'development',\n",
       " 'attrs',\n",
       " '浅拷贝',\n",
       " 'txHash',\n",
       " '基本',\n",
       " 'copyObj',\n",
       " 'node4',\n",
       " '消息',\n",
       " '传递数据',\n",
       " 'cmd',\n",
       " 'defa',\n",
       " 'Helpers',\n",
       " '发布者',\n",
       " '静态',\n",
       " 'map',\n",
       " 'objClone',\n",
       " 'setModelValue',\n",
       " '目录',\n",
       " 'HTTP',\n",
       " '东西',\n",
       " 'isArray',\n",
       " 'split',\n",
       " '关系',\n",
       " '资金',\n",
       " 'List',\n",
       " '引用数据类型',\n",
       " '选项',\n",
       " '教程',\n",
       " '思想',\n",
       " '玩家',\n",
       " 'github',\n",
       " 'tml',\n",
       " '前言',\n",
       " '新手入门',\n",
       " '事件触发',\n",
       " 'list3',\n",
       " 'obj3',\n",
       " 'copy',\n",
       " 'ttp',\n",
       " 'CLI',\n",
       " 'color',\n",
       " 'color1',\n",
       " '快捷键',\n",
       " 'ES6',\n",
       " '淘宝',\n",
       " '命令行工具',\n",
       " 'baidu',\n",
       " 'event1',\n",
       " 'objA',\n",
       " 'message',\n",
       " '原创',\n",
       " 'env',\n",
       " 'port',\n",
       " 'python',\n",
       " '基础数据类型',\n",
       " '大量',\n",
       " '元素',\n",
       " 'outdb',\n",
       " 'else',\n",
       " '有点',\n",
       " 'propE',\n",
       " 'example',\n",
       " 'nonce',\n",
       " 'customClone',\n",
       " 'listen',\n",
       " '技术原理',\n",
       " '5e',\n",
       " 'www',\n",
       " 'throw',\n",
       " '父子',\n",
       " 'cjg',\n",
       " '意思',\n",
       " '链接',\n",
       " '课程',\n",
       " '浏览器',\n",
       " 'com',\n",
       " '输入框',\n",
       " 'indexOf',\n",
       " 'true',\n",
       " 'objB',\n",
       " 'Name',\n",
       " '结果',\n",
       " 'php',\n",
       " 'addr',\n",
       " 'notEqual',\n",
       " 'SSE',\n",
       " '网站',\n",
       " 'NPM',\n",
       " '原生',\n",
       " 'childMsg2',\n",
       " 'applet',\n",
       " '部分',\n",
       " '脚本',\n",
       " '后台',\n",
       " 'aaa',\n",
       " 'production',\n",
       " '解决方案',\n",
       " '次数',\n",
       " 'array2',\n",
       " '开发环境',\n",
       " '地方',\n",
       " 'moneyIncomePay',\n",
       " '使用细节',\n",
       " '连胜',\n",
       " '框架',\n",
       " '人们',\n",
       " '驼峰',\n",
       " '成本',\n",
       " '篇文章',\n",
       " 'txt',\n",
       " '空间',\n",
       " '内存空间',\n",
       " 'solidity',\n",
       " 'contract',\n",
       " '命令行',\n",
       " 'localhost',\n",
       " '技术栈',\n",
       " 'run',\n",
       " 'loader',\n",
       " 'txn',\n",
       " '工程师',\n",
       " '个字符',\n",
       " 'dev',\n",
       " '范围',\n",
       " '性能',\n",
       " 'ram',\n",
       " 'demo',\n",
       " 'EOS',\n",
       " '示例',\n",
       " 'trinoc',\n",
       " '启动服务',\n",
       " '层级',\n",
       " 'uninstall',\n",
       " 'assert',\n",
       " 'items',\n",
       " '作者',\n",
       " '嵌套',\n",
       " 'computed',\n",
       " 'window',\n",
       " 'dep',\n",
       " '文档',\n",
       " 'string',\n",
       " '长连接',\n",
       " 'getIn',\n",
       " 'use',\n",
       " 'childMsg1',\n",
       " 'res',\n",
       " 'instance',\n",
       " '列表',\n",
       " 'Server',\n",
       " '错误',\n",
       " 'toggle',\n",
       " '分数',\n",
       " 'plugin',\n",
       " 'immutability',\n",
       " '环境',\n",
       " '配置项',\n",
       " 'robomongod',\n",
       " 'publish',\n",
       " 'proportion',\n",
       " 'typeof',\n",
       " 'age',\n",
       " '状态机',\n",
       " '向子',\n",
       " '规格',\n",
       " '网络',\n",
       " 'Dynamic',\n",
       " 'then',\n",
       " '发送消息',\n",
       " '网络请求',\n",
       " 'selector',\n",
       " 'less',\n",
       " 'public',\n",
       " 'helper',\n",
       " 'extract',\n",
       " '下载地址',\n",
       " 'ctrl',\n",
       " '中划线',\n",
       " '老板',\n",
       " 'websocket',\n",
       " 'ckPlugin',\n",
       " 'extend',\n",
       " 'x225263280000000000000000000000000000000000000000000000000000000000000001',\n",
       " 'wei',\n",
       " '面向',\n",
       " 'attributes',\n",
       " 'cloneObj',\n",
       " 'file',\n",
       " '函数返回',\n",
       " 'ENV',\n",
       " 'download',\n",
       " 'Value',\n",
       " '知识点',\n",
       " 'component',\n",
       " 'pm2',\n",
       " 'title',\n",
       " 'TCP',\n",
       " '核心',\n",
       " 'conf',\n",
       " 'mini',\n",
       " '区别',\n",
       " 'thread',\n",
       " '安全性',\n",
       " '切图',\n",
       " 'false',\n",
       " 'npmjs',\n",
       " 'DApp',\n",
       " '对方',\n",
       " 'html',\n",
       " '源地址',\n",
       " 'NODE',\n",
       " '方向',\n",
       " 'createDocumentFragment',\n",
       " '架构',\n",
       " 'exe',\n",
       " '和尚',\n",
       " 'java',\n",
       " 'cket',\n",
       " 'blog',\n",
       " 'app',\n",
       " '我会',\n",
       " '感觉',\n",
       " 'Websocket',\n",
       " 'notify',\n",
       " '队列',\n",
       " '账户',\n",
       " '计时器',\n",
       " '都行',\n",
       " 'api',\n",
       " 'admin',\n",
       " '选择器',\n",
       " 'x771602f700000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002',\n",
       " '出镜',\n",
       " '效率',\n",
       " 'sync',\n",
       " '字节',\n",
       " 'Php',\n",
       " '单向',\n",
       " 'vuex',\n",
       " 'arg',\n",
       " '服务器端',\n",
       " 'Function',\n",
       " 'returns',\n",
       " '过滤器',\n",
       " '宝宝',\n",
       " '管理系统',\n",
       " 'events',\n",
       " 'pkey',\n",
       " 'ttributes',\n",
       " '重命名',\n",
       " '全双工',\n",
       " 'caidan1',\n",
       " 'boolean',\n",
       " 'bind',\n",
       " '进阶',\n",
       " 'i++',\n",
       " '通信协议',\n",
       " 'Web应用程序',\n",
       " '消息推送',\n",
       " '数字',\n",
       " '整体',\n",
       " 'gasLimit',\n",
       " '个人',\n",
       " 'Math',\n",
       " '钱包',\n",
       " '数据类型',\n",
       " '谢谢',\n",
       " 'provider',\n",
       " '文末',\n",
       " 'text',\n",
       " 'list2',\n",
       " '处理方法',\n",
       " 'splitChunks',\n",
       " '安装版',\n",
       " '和子',\n",
       " 'job',\n",
       " '本局',\n",
       " '费用',\n",
       " '官网',\n",
       " '字段',\n",
       " 'game',\n",
       " 'Boolean',\n",
       " '实例',\n",
       " 'replace',\n",
       " 'removeAttribute',\n",
       " '目标值',\n",
       " '花费',\n",
       " 'honeyBadger8',\n",
       " 'watch',\n",
       " 'ABI',\n",
       " '名字',\n",
       " 'pow',\n",
       " 'event',\n",
       " '函数定义',\n",
       " '朋友',\n",
       " '指向',\n",
       " '文件名',\n",
       " 'while',\n",
       " '点击事件',\n",
       " 'etc',\n",
       " 'zht',\n",
       " 'mongoimport',\n",
       " 'addUserBalance',\n",
       " '端口',\n",
       " 'CVM',\n",
       " 'deploy',\n",
       " '开源',\n",
       " '语句',\n",
       " '代币',\n",
       " '生效',\n",
       " '比特币开发',\n",
       " '同学',\n",
       " '表格',\n",
       " '命名',\n",
       " 'Star',\n",
       " 'clicli',\n",
       " 'privateKey',\n",
       " 'list1',\n",
       " '博客',\n",
       " '路径',\n",
       " 'data1',\n",
       " '案例',\n",
       " 'ddEventListener',\n",
       " '场景',\n",
       " '移动端',\n",
       " '比特币',\n",
       " 'walk',\n",
       " '样式',\n",
       " '关键',\n",
       " 'oldVal',\n",
       " 'querySelector',\n",
       " 'sendObj',\n",
       " '开源框架',\n",
       " '布尔值',\n",
       " 'iojs',\n",
       " 'Promise',\n",
       " 'upsert',\n",
       " 'propA',\n",
       " 'c741a1218efc0144adc1749125a9ca53134b06353a8e4ef72afa07c50fb59647ff8b8895b75795b0',\n",
       " 'Windows',\n",
       " 'Creates',\n",
       " '标签',\n",
       " 'sses',\n",
       " '步骤',\n",
       " '个人信息',\n",
       " '想象',\n",
       " '温度',\n",
       " 'DNS',\n",
       " 'Instead',\n",
       " 'c4f215fcae9b21',\n",
       " 'current',\n",
       " 'end',\n",
       " 'brief',\n",
       " 'propC',\n",
       " 'grunt',\n",
       " 'wangnian',\n",
       " '同类',\n",
       " '大写',\n",
       " 'myOtherTests',\n",
       " 'NamedMod',\n",
       " '本地开发',\n",
       " 'connect',\n",
       " 'cssExtractTextWeb',\n",
       " 'functions',\n",
       " '发布订阅',\n",
       " 'define',\n",
       " 'that',\n",
       " 'itself',\n",
       " 'gasUsed',\n",
       " 'Nginx',\n",
       " 'Port',\n",
       " '局部',\n",
       " '货币',\n",
       " 'size',\n",
       " 'toJS',\n",
       " 'b346755bca0',\n",
       " '种类',\n",
       " '版本号',\n",
       " '微信公众平台',\n",
       " '云小',\n",
       " '好气',\n",
       " '时代',\n",
       " 'out',\n",
       " '标准',\n",
       " '输方',\n",
       " 'mocha',\n",
       " 'constant',\n",
       " '版本更新',\n",
       " '测试框架',\n",
       " 'down',\n",
       " '详细教程',\n",
       " '信息管理',\n",
       " '包管理',\n",
       " 'namedMod',\n",
       " 'bool',\n",
       " 'Handle',\n",
       " 'NodeJs',\n",
       " 'services',\n",
       " 'Buffer',\n",
       " 'JQuery',\n",
       " '传送门',\n",
       " 'article',\n",
       " 'R3',\n",
       " '差点',\n",
       " '梅花香',\n",
       " '通路',\n",
       " 'nodist',\n",
       " 'undefined',\n",
       " '世界',\n",
       " '能力',\n",
       " 'json数据',\n",
       " '大神',\n",
       " 'encourages',\n",
       " 'lazy',\n",
       " '赢方',\n",
       " '团队',\n",
       " 'reverse',\n",
       " '优点',\n",
       " '网络层',\n",
       " 'devDependencies',\n",
       " 'event2',\n",
       " 'WxSocketIO',\n",
       " '张小龙',\n",
       " '传统',\n",
       " '可视化',\n",
       " 'react',\n",
       " '精心',\n",
       " '质量',\n",
       " '图标',\n",
       " '新人',\n",
       " 'uglifyjs',\n",
       " '时会',\n",
       " 'required',\n",
       " 'serialize',\n",
       " 'ECMAScript',\n",
       " 'programming',\n",
       " '数据流',\n",
       " 'msc',\n",
       " '宝贵意见',\n",
       " 'WEB',\n",
       " 'send',\n",
       " '回滚',\n",
       " 'isAttrs',\n",
       " '作用域',\n",
       " 'ethereum',\n",
       " 'ipfs',\n",
       " 'caidan4',\n",
       " 'createServer',\n",
       " 'application',\n",
       " 'origin',\n",
       " 'toNumber',\n",
       " 'scope',\n",
       " 'newServer',\n",
       " '普通',\n",
       " 'gitHub',\n",
       " 'keccak',\n",
       " 'esPlugin',\n",
       " 'Python',\n",
       " '文件路径',\n",
       " 'raw',\n",
       " '树形',\n",
       " 'fromJS',\n",
       " 'x29f2f640',\n",
       " '密钥对',\n",
       " '增量',\n",
       " 'oschina',\n",
       " 'follow',\n",
       " '新手',\n",
       " '单据',\n",
       " '小心',\n",
       " 'getTransactionCount',\n",
       " 'color4',\n",
       " '礼包',\n",
       " '关心',\n",
       " '商业',\n",
       " 'xml',\n",
       " 'serializedTx',\n",
       " '节省',\n",
       " '金额',\n",
       " 'propB',\n",
       " '领路人',\n",
       " '注册表',\n",
       " 'with',\n",
       " '包将',\n",
       " 'repo',\n",
       " '由子',\n",
       " '区域',\n",
       " '玩味',\n",
       " '终端',\n",
       " '实时同步',\n",
       " 'Avoid',\n",
       " '观察者',\n",
       " 'robomongo',\n",
       " '创建数据库',\n",
       " '套件',\n",
       " '速记',\n",
       " '身份',\n",
       " 'concat',\n",
       " 'vendor',\n",
       " '表达式',\n",
       " '小游戏',\n",
       " 'C#开发',\n",
       " '频率',\n",
       " '新旧',\n",
       " 'concatenateMod',\n",
       " 'nave',\n",
       " 'caidan3',\n",
       " 'pyethapp',\n",
       " '3c',\n",
       " 'mongo',\n",
       " 'without',\n",
       " '线索',\n",
       " '入口',\n",
       " '证据',\n",
       " 'Prop',\n",
       " 'city',\n",
       " 'asOwnProperty',\n",
       " '个人观点',\n",
       " '小娜',\n",
       " '命名规范',\n",
       " '占位符',\n",
       " '服务器环境',\n",
       " 'nodebrew',\n",
       " 'addSub',\n",
       " '新建',\n",
       " '自定义方法',\n",
       " 'such',\n",
       " '按钮',\n",
       " '三观',\n",
       " '响应式',\n",
       " 'Mac',\n",
       " '安装使用',\n",
       " 'Medium',\n",
       " '管理工具',\n",
       " 'MobileIMSDK',\n",
       " '管理员',\n",
       " '代表',\n",
       " 'v8',\n",
       " 'details',\n",
       " 'say',\n",
       " '密钥管理',\n",
       " '信号',\n",
       " 'React',\n",
       " '大佬',\n",
       " 'keccak256',\n",
       " '上线',\n",
       " 'warn',\n",
       " 'signature',\n",
       " '项目文件',\n",
       " '数据表',\n",
       " '办法',\n",
       " '百度地图',\n",
       " 'SymbolVue',\n",
       " 'Mongo',\n",
       " 'installnpm',\n",
       " '稳定版',\n",
       " '强制性',\n",
       " '屏幕',\n",
       " 'version',\n",
       " 'techniques',\n",
       " 'mergeIn',\n",
       " '中文',\n",
       " 'color3',\n",
       " '人群',\n",
       " 'invoked',\n",
       " '面试官',\n",
       " 'evaluation',\n",
       " '硬编码',\n",
       " '泥马',\n",
       " 'directly',\n",
       " '结缘',\n",
       " '书写',\n",
       " '实际',\n",
       " '消耗',\n",
       " 'runoob',\n",
       " '上例',\n",
       " '标题',\n",
       " '微信开发',\n",
       " 'rity',\n",
       " '对象存储',\n",
       " 'started',\n",
       " 'strings',\n",
       " '共识机制',\n",
       " 'words',\n",
       " '交流学习',\n",
       " 'Homebrew',\n",
       " '博客地址',\n",
       " '流水',\n",
       " '画面',\n",
       " 'renders',\n",
       " '学校',\n",
       " 'E盘',\n",
       " 'myTruffleTest',\n",
       " 'will',\n",
       " 'Html',\n",
       " 'ECMA',\n",
       " '工具集',\n",
       " '样例',\n",
       " '共享内存',\n",
       " 'tree',\n",
       " '日志',\n",
       " 'memberof',\n",
       " 'js对象',\n",
       " '小结',\n",
       " '直接访问',\n",
       " '2d',\n",
       " '即时通讯开发',\n",
       " 'wdlhao',\n",
       " 'height',\n",
       " 'getBalance',\n",
       " 'Pomelo',\n",
       " 'Web开发',\n",
       " 'rray',\n",
       " 'info',\n",
       " 'overwritten',\n",
       " '损耗',\n",
       " 'since',\n",
       " 'Chrome',\n",
       " 'UTXO',\n",
       " 'a307baded0b0672745691358e3ee6',\n",
       " 'ddSub',\n",
       " '业务',\n",
       " '证明',\n",
       " '智网',\n",
       " 'jQuery',\n",
       " '端口号',\n",
       " '指针',\n",
       " '项目部署',\n",
       " 'Netty',\n",
       " '网络同步',\n",
       " '余地',\n",
       " 'tips',\n",
       " 'handle',\n",
       " 'propD',\n",
       " 'qcloud',\n",
       " 'geth',\n",
       " '背景色',\n",
       " 'f72',\n",
       " '命令提示符',\n",
       " '编程',\n",
       " 'API函数',\n",
       " 'getter',\n",
       " 'BigNumbers',\n",
       " '细节',\n",
       " 'ck2',\n",
       " '权力',\n",
       " '网络状态',\n",
       " 'weixin',\n",
       " 'router',\n",
       " '编辑',\n",
       " 'fields',\n",
       " 'Socket通信',\n",
       " '盘符',\n",
       " '有所',\n",
       " '账号',\n",
       " '对子',\n",
       " 'Sho',\n",
       " '开发模式',\n",
       " 'some',\n",
       " '标明',\n",
       " '版本信息',\n",
       " '标志',\n",
       " 'property',\n",
       " '云技术',\n",
       " 'About',\n",
       " 'emit',\n",
       " '整理',\n",
       " 'sign',\n",
       " '数据段',\n",
       " 'simpler',\n",
       " 'Comet',\n",
       " 'lightwallet',\n",
       " 'ExtractTextWeb',\n",
       " '格式',\n",
       " 'windows系统',\n",
       " '工厂',\n",
       " '业务需求',\n",
       " 'android',\n",
       " '开发教程',\n",
       " ...]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 可以进行转换\n",
    "cv_m.vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.6061358 , 0.        , 0.6061358 , 0.6061358 , 0.45198512,\n",
       "       0.78845736, 1.01160091, 1.01160091, 1.01160091, 0.78845736,\n",
       "       1.29928298, 1.70474809, 0.31845373, 1.01160091, 0.78845736,\n",
       "       0.45198512, 0.78845736, 0.78845736, 0.45198512, 1.70474809])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idfModel.idf.toArray()[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# IDF对CV结果进行计算TFIDF\n",
    "from pyspark.ml.feature import IDFModel\n",
    "idf_model = IDFModel.load(\"hdfs://hadoop-master:9000/headlines/models/testIDF.model\")\n",
    "tfidf_res = idf_model.transform(cv_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+----------+--------------------+--------------------+--------------------+\n",
      "|article_id|channel_id|               words|       countFeatures|         idfFeatures|\n",
      "+----------+----------+--------------------+--------------------+--------------------+\n",
      "|         1|        17|[Vue, props, 用法, ...|(1265,[0,1,3,4,5,...|(1265,[0,1,3,4,5,...|\n",
      "|         2|        17|[vue, 响应式, 原理, mo...|(1265,[0,1,2,3,4,...|(1265,[0,1,2,3,4,...|\n",
      "|         3|        17|[JavaScript, 浅拷贝,...|(1265,[0,1,5,7,12...|(1265,[0,1,5,7,12...|\n",
      "|         4|        17|[vue2, vuex, elem...|(1265,[1,2,4,9,12...|(1265,[1,2,4,9,12...|\n",
      "|         5|        17|[immutability, Re...|(1265,[1,3,4,5,6,...|(1265,[1,3,4,5,6,...|\n",
      "|         6|        17|[node, npm, cnpm,...|(1265,[1,2,9,12,1...|(1265,[1,2,9,12,1...|\n",
      "|         7|        17|[Web, 工程师, 以太坊, 入...|(1265,[1,2,3,4,6,...|(1265,[1,2,3,4,6,...|\n",
      "|         8|        17|[Web, pa, api, we...|(1265,[1,2,9,30,3...|(1265,[1,2,9,30,3...|\n",
      "|         9|        17|[vue, 中用, 数据驱动, 视...|(1265,[0,1,4,8,14...|(1265,[0,1,4,8,14...|\n",
      "|        10|        17|[程序, WebSocket, 长...|(1265,[0,1,3,8,12...|(1265,[0,1,3,8,12...|\n",
      "+----------+----------+--------------------+--------------------+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tfidf_res.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1265词的 {索引 以及 权重}\n",
    "def func(partition):\n",
    "    TOPK = 20\n",
    "    for row in partition:\n",
    "        # 找到索引与IDF值并进行排序\n",
    "        _ = list(zip(row.idfFeatures.indices, row.idfFeatures.values))\n",
    "        _ = sorted(_, key=lambda x: x[1], reverse=True)\n",
    "        result = _[:TOPK]\n",
    "        for word_index, tfidf in result:\n",
    "            yield row.article_id, row.channel_id, int(word_index), round(float(tfidf), 4)\n",
    "kewords_tfidf = tfidf_res.rdd.mapPartitions(func).toDF(['article_id', 'channel_id', 'index', 'weights'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+----------+-----+-------+\n",
      "|article_id|channel_id|index|weights|\n",
      "+----------+----------+-----+-------+\n",
      "|         1|        17|   19|68.1899|\n",
      "|         1|        17|   10|58.4677|\n",
      "|         1|        17|    8|55.6381|\n",
      "|         1|        17|   26|52.8472|\n",
      "|         1|        17|   39|39.2092|\n",
      "|         1|        17|   27|31.1828|\n",
      "|         1|        17|   55|30.6855|\n",
      "|         1|        17|   58|30.6855|\n",
      "|         1|        17|   66| 27.276|\n",
      "|         1|        17|  138|15.3427|\n",
      "|         1|        17|  139|15.3427|\n",
      "|         1|        17|  150| 13.638|\n",
      "|         1|        17|  171| 13.638|\n",
      "|         1|        17|  175| 13.638|\n",
      "|         1|        17|    4|11.2996|\n",
      "|         1|        17|  206|10.2285|\n",
      "|         1|        17|  239| 8.5237|\n",
      "|         1|        17|  267| 8.5237|\n",
      "|         1|        17|   33| 8.1357|\n",
      "|         1|        17|  295|  6.819|\n",
      "+----------+----------+-----+-------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "kewords_tfidf.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 利用keywordsIndex = ktt.spark.sql(\"select keyword, index idx from idf_keywords_values\")中标，知道索引对应的词\n",
    "idf_keywords_values = oa.spark.sql(\"select keyword, index idx from idf_keywords_values\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "keyword_str_tfidf = kewords_tfidf.join(idf_keywords_values, idf_keywords_values.idx==kewords_tfidf.index).select([\"article_id\", \"channel_id\", \"keyword\", \"weights\"])\n",
    "\n",
    "keyword_str_tfidf.show()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# texrank\n",
    "# 分词\n",
    "def textrank(partition):\n",
    "    import os\n",
    "\n",
    "    import jieba\n",
    "    import jieba.analyse\n",
    "    import jieba.posseg as pseg\n",
    "    import codecs\n",
    "\n",
    "    abspath = \"/root/words\"\n",
    "\n",
    "    # 结巴加载用户词典\n",
    "    userDict_path = os.path.join(abspath, \"ITKeywords.txt\")\n",
    "    jieba.load_userdict(userDict_path)\n",
    "\n",
    "    # 停用词文本\n",
    "    stopwords_path = os.path.join(abspath, \"stopwords.txt\")\n",
    "\n",
    "    def get_stopwords_list():\n",
    "        \"\"\"返回stopwords列表\"\"\"\n",
    "        stopwords_list = [i.strip()\n",
    "                          for i in codecs.open(stopwords_path).readlines()]\n",
    "        return stopwords_list\n",
    "\n",
    "    # 所有的停用词列表\n",
    "    stopwords_list = get_stopwords_list()\n",
    "\n",
    "    class TextRank(jieba.analyse.TextRank):\n",
    "        def __init__(self, window=20, word_min_len=2):\n",
    "            super(TextRank, self).__init__()\n",
    "            self.span = window  # 窗口大小\n",
    "            self.word_min_len = word_min_len  # 单词的最小长度\n",
    "            # 要保留的词性，根据jieba github ，具体参见https://github.com/baidu/lac\n",
    "            self.pos_filt = frozenset(\n",
    "                ('n', 'x', 'eng', 'f', 's', 't', 'nr', 'ns', 'nt', \"nw\", \"nz\", \"PER\", \"LOC\", \"ORG\"))\n",
    "\n",
    "        def pairfilter(self, wp):\n",
    "            \"\"\"过滤条件，返回True或者False\"\"\"\n",
    "\n",
    "            if wp.flag == \"eng\":\n",
    "                if len(wp.word) <= 2:\n",
    "                    return False\n",
    "\n",
    "            if wp.flag in self.pos_filt and len(wp.word.strip()) >= self.word_min_len \\\n",
    "                    and wp.word.lower() not in stopwords_list:\n",
    "                return True\n",
    "    # TextRank过滤窗口大小为5，单词最小为2\n",
    "    textrank_model = TextRank(window=5, word_min_len=2)\n",
    "    allowPOS = ('n', \"x\", 'eng', 'nr', 'ns', 'nt', \"nw\", \"nz\", \"c\")\n",
    "\n",
    "    for row in partition:\n",
    "        tags = textrank_model.textrank(row.sentence, topK=20, withWeight=True, allowPOS=allowPOS, withFlag=False)\n",
    "        for tag in tags:\n",
    "            yield row.article_id, row.channel_id, tag[0], tag[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "textrank = article_data.rdd.mapPartitions(textrank).toDF([\"article_id\", \"channel_id\", \"keyword\", \"textrank\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
